import numpy as np 
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt 
from sklearn.model_selection import cross_val_score, train_test_split, GridSearchCV
from sklearn.linear_model import LinearRegression, Lasso
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, AdaBoostRegressor, VotingRegressor
import xgboost as xgb
import warnings

# Ignore all warnings
warnings.filterwarnings("ignore")

X = pd.read_csv("concrete.csv")

X.head()

X.describe()

X.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1030 entries, 0 to 1029
Data columns (total 9 columns):
 #   Column        Non-Null Count  Dtype  
---  ------        --------------  -----  
 0   cement        1030 non-null   float64
 1   slag          1030 non-null   float64
 2   ash           1030 non-null   float64
 3   water         1030 non-null   float64
 4   superplastic  1030 non-null   float64
 5   coarseagg     1030 non-null   float64
 6   fineagg       1030 non-null   float64
 7   age           1030 non-null   int64  
 8   strength      1030 non-null   float64
dtypes: float64(8), int64(1)
memory usage: 72.6 KB

X_features = X.iloc[:, :-1]  # Selects all columns except the last one
target_col = ["strength"]  
X_features

def plot_sns_corr_regre(X_features, target_col):
    g = sns.PairGrid(X_features)
    g.map_upper(sns.scatterplot)
    g.map_lower(sns.kdeplot)
    g.map_diag(sns.kdeplot, lw=3, legend=False)
    g.fig.suptitle('Pairwise data relationships', y=1.01)
    plt.show()
plot_sns_corr_regre(X_features, 'concrete_compressive_strength')

X.hist(figsize=(10,10))
plt.show()

sns.pairplot(X)
plt.show()

corr = X.corr()
mask = np.zeros_like(corr, dtype=bool)
mask[np.triu_indices_from(mask)] = True
f, ax = plt.subplots(figsize=(12, 10))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
sns.heatmap(corr, mask=mask,cmap=cmap, vmax=.9, center=0, square=True, linewidths=.5, annot=True,cbar_kws={"shrink": .5})
plt.show()

def plot_jointplots(data, target_column_name):
    feature_names = data.columns.tolist()
    feature_names.remove(target_column_name)

    for feature_name in feature_names:
        sns.jointplot(x=feature_name, y=target_column_name, data=data, kind="reg")
        plt.show()

plot_jointplots(X, 'strength')

sns.histplot(X['strength'], bins=30, color='red', alpha=0.7)

<Axes: xlabel='strength', ylabel='Count'>

cols = X.columns

plt.figure(figsize = (16, 20))
plotnumber = 1

for i in range(1, len(cols)):
    if plotnumber <= 9:
        ax = plt.subplot(3, 3, plotnumber)
        sns.boxplot(y = cols[i], data = X, ax = ax)
        plt.title(f"\n{cols[i]} \n", fontsize = 20)
        
    plotnumber += 1

plt.tight_layout()
plt.show()

# Import necessary libraries

# Assuming df_features and df_strength are your features and target DataFrames
X = X_features
y = X.iloc[:, -1]  # Selects the last column which is 'strength'


# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Initialize models with hyperparameter grids
param_grids = {
    'Linear Regression': {
        'fit_intercept': [True, False],
        'copy_X': [True, False]
    },
    'Lasso Regression': {
        'alpha': [0.1, 1.0, 10.0],
        'max_iter': [1000, 5000, 10000]
    },
    'Decision Tree': {
        'max_depth': [3, 5, 10, None],
        'min_samples_split': [2, 5, 10]
    },
    'Random Forest': {
        'n_estimators': [100, 200, 500],
        'max_depth': [3, 5, 10, None],
        'min_samples_split': [2, 5, 10]
    },
    'Ada Boost': {
        'n_estimators': [50, 100, 200],
        'learning_rate': [0.01, 0.1, 1.0]
    },
    'Gradient Boost': {
        'n_estimators': [100, 200, 500],
        'learning_rate': [0.01, 0.1, 0.2],
        'max_depth': [3, 5, 10]
    },
    'XGBoost': {
        'n_estimators': [100, 200, 500],
        'learning_rate': [0.01, 0.1, 0.2],
        'max_depth': [3, 5, 10]
    }
}

# Models initialization
models = {
    'Linear Regression': LinearRegression(),
    'Lasso Regression': Lasso(),
    'Decision Tree': DecisionTreeRegressor(random_state=42),
    'Random Forest': RandomForestRegressor(random_state=42),
    'Ada Boost': AdaBoostRegressor(random_state=42),
    'Gradient Boost': GradientBoostingRegressor(random_state=42),
    'XGBoost': xgb.XGBRegressor(random_state=42)
}

# Hyperparameter tuning using GridSearchCV
best_models = {}
for name, model in models.items():
    print(f"Running GridSearchCV for {name}...")
    grid_search = GridSearchCV(estimator=model, param_grid=param_grids[name], cv=5, scoring='neg_mean_squared_error')
    grid_search.fit(X_train, y_train)
    best_models[name] = grid_search.best_estimator_
    print(f"Best parameters for {name}: {grid_search.best_params_}")
    print(f"Best RMSE for {name}: {np.sqrt(-grid_search.best_score_):.4f}")
    print("-" * 50)

# Create Voting Regressor with the best models
voting_regressor = VotingRegressor(estimators=[
    ('lr', best_models['Linear Regression']),
    ('lasso', best_models['Lasso Regression']),
    ('dt', best_models['Decision Tree']),
    ('rf', best_models['Random Forest']),
    ('ada', best_models['Ada Boost']),
    ('gb', best_models['Gradient Boost']),
    ('xgb', best_models['XGBoost'])
])

# Fit the voting regressor and evaluate it
voting_regressor.fit(X_train, y_train)
voting_scores = cross_val_score(voting_regressor, X_train, y_train, cv=5, scoring='neg_mean_squared_error')
voting_rmse = np.sqrt(-voting_scores)
print(f"Voting Regressor - RMSE: {voting_rmse.mean():.4f} (+/- {voting_rmse.std():.4f})")

Running GridSearchCV for Linear Regression...
Best parameters for Linear Regression: {'copy_X': True, 'fit_intercept': True}
Best RMSE for Linear Regression: 0.0000
--------------------------------------------------
Running GridSearchCV for Lasso Regression...
Best parameters for Lasso Regression: {'alpha': 0.1, 'max_iter': 1000}
Best RMSE for Lasso Regression: 0.0043
--------------------------------------------------
Running GridSearchCV for Decision Tree...
Best parameters for Decision Tree: {'max_depth': 10, 'min_samples_split': 2}
Best RMSE for Decision Tree: 0.1047
--------------------------------------------------
Running GridSearchCV for Random Forest...
Best parameters for Random Forest: {'max_depth': 5, 'min_samples_split': 2, 'n_estimators': 500}
Best RMSE for Random Forest: 0.4707
--------------------------------------------------
Running GridSearchCV for Ada Boost...
Best parameters for Ada Boost: {'learning_rate': 0.1, 'n_estimators': 100}
Best RMSE for Ada Boost: 2.3559
--------------------------------------------------
Running GridSearchCV for Gradient Boost...
Best parameters for Gradient Boost: {'learning_rate': 0.2, 'max_depth': 3, 'n_estimators': 500}
Best RMSE for Gradient Boost: 0.1045
--------------------------------------------------
Running GridSearchCV for XGBoost...
Best parameters for XGBoost: {'learning_rate': 0.1, 'max_depth': 5, 'n_estimators': 200}
Best RMSE for XGBoost: 0.1004
--------------------------------------------------
Voting Regressor - RMSE: 0.3601 (+/- 0.0514)

from sklearn.model_selection import RandomizedSearchCV

param_grids = {
    'XGBoost': {
        'n_estimators': [100, 200, 500, 1000],
        'learning_rate': [0.01, 0.05, 0.1, 0.2],
        'max_depth': [3, 5, 7, 10],
        'subsample': [0.6, 0.8, 1.0],
        'colsample_bytree': [0.6, 0.8, 1.0],
        'gamma': [0, 0.1, 0.2, 0.3]
    }
}

# RandomizedSearchCV for XGBoost
xgb_random_search = RandomizedSearchCV(estimator=xgb.XGBRegressor(random_state=42),
                                       param_distributions=param_grids['XGBoost'],
                                       n_iter=100, cv=5, scoring='neg_mean_squared_error', 
                                       random_state=42, n_jobs=-1)
xgb_random_search.fit(X_train, y_train)
print(f"Best parameters for XGBoost: {xgb_random_search.best_params_}")
print(f"Best RMSE for XGBoost: {np.sqrt(-xgb_random_search.best_score_):.4f}")

Best parameters for XGBoost: {'subsample': 1.0, 'n_estimators': 500, 'max_depth': 3, 'learning_rate': 0.05, 'gamma': 0, 'colsample_bytree': 1.0}
Best RMSE for XGBoost: 0.0988

Attribute	Unit	Description
Cement	kg/m³	Amount of cement in the mixture
Blast Furnace Slag	kg/m³	Amount of blast furnace slag
Fly Ash	kg/m³	Amount of fly ash
Water	kg/m³	Amount of water
Superplasticizer	kg/m³	Amount of superplasticizer
Coarse Aggregate	kg/m³	Amount of coarse aggregate
Fine Aggregate	kg/m³	Amount of fine aggregate
Age	days	Curing age of the concrete
Compressive Strength	MPa	Target variable - concrete's compressive strength

	cement	slag	ash	water	superplastic	coarseagg	fineagg	age	strength
0	141.3	212.0	0.0	203.5	0.0	971.8	748.5	28	29.89
1	168.9	42.2	124.3	158.3	10.8	1080.8	796.2	14	23.51
2	250.0	0.0	95.7	187.4	5.5	956.9	861.2	28	29.22
3	266.0	114.0	0.0	228.0	0.0	932.0	670.0	28	45.85
4	154.8	183.4	0.0	193.3	9.1	1047.4	696.7	28	18.29

	cement	slag	ash	water	superplastic	coarseagg	fineagg	age	strength
count	1030.000000	1030.000000	1030.000000	1030.000000	1030.000000	1030.000000	1030.000000	1030.000000	1030.000000
mean	281.167864	73.895825	54.188350	181.567282	6.204660	972.918932	773.580485	45.662136	35.817961
std	104.506364	86.279342	63.997004	21.354219	5.973841	77.753954	80.175980	63.169912	16.705742
min	102.000000	0.000000	0.000000	121.800000	0.000000	801.000000	594.000000	1.000000	2.330000
25%	192.375000	0.000000	0.000000	164.900000	0.000000	932.000000	730.950000	7.000000	23.710000
50%	272.900000	22.000000	0.000000	185.000000	6.400000	968.000000	779.500000	28.000000	34.445000
75%	350.000000	142.950000	118.300000	192.000000	10.200000	1029.400000	824.000000	56.000000	46.135000
max	540.000000	359.400000	200.100000	247.000000	32.200000	1145.000000	992.600000	365.000000	82.600000

	cement	slag	ash	water	superplastic	coarseagg	fineagg	age
0	141.3	212.0	0.0	203.5	0.0	971.8	748.5	28
1	168.9	42.2	124.3	158.3	10.8	1080.8	796.2	14
2	250.0	0.0	95.7	187.4	5.5	956.9	861.2	28
3	266.0	114.0	0.0	228.0	0.0	932.0	670.0	28
4	154.8	183.4	0.0	193.3	9.1	1047.4	696.7	28
...	...	...	...	...	...	...	...	...
1025	135.0	0.0	166.0	180.0	10.0	961.0	805.0	28
1026	531.3	0.0	0.0	141.8	28.2	852.1	893.7	3
1027	276.4	116.0	90.3	179.6	8.9	870.1	768.3	28
1028	342.0	38.0	0.0	228.0	0.0	932.0	670.0	270
1029	540.0	0.0	0.0	173.0	0.0	1125.0	613.0	7

Model	Best Parameters	Best RMSE	Accuracy (%)	Interpretation
Linear Regression	{'copy_X': True, 'fit_intercept': True}	0.0000	100.00	Perfectly fits the data. Could indicate overfitting or that the data is very linear.
Lasso Regression	{'alpha': 0.1, 'max_iter': 1000}	0.0043	99.99	Low RMSE suggests regularization is helping but the data may still be highly linear.
Decision Tree	{'max_depth': 10, 'min_samples_split': 2}	0.1047	99.87	Performs reasonably well. The higher RMSE compared to linear models suggests more data complexity.
Random Forest	{'max_depth': 5, 'min_samples_split': 2, 'n_estimators': 500}	0.4707	99.43	Higher RMSE suggests the ensemble method is not capturing the data's complexity as well as the simpler models.
AdaBoost	{'learning_rate': 0.1, 'n_estimators': 100}	2.3559	97.13	Highest RMSE among all models. Indicates it may not be well-suited for this dataset or needs more tuning.
Gradient Boosting	{'learning_rate': 0.2, 'max_depth': 3, 'n_estimators': 500}	0.1045	99.87	Performs similarly to Decision Tree with a low RMSE, capturing non-linearities effectively.
XGBoost	{'subsample': 1.0, 'n_estimators': 500, 'max_depth': 3, 'learning_rate': 0.05, 'gamma': 0, 'colsample_bytree': 1.0}	0.0988	99.88	XGBoost outperforms Gradient Boosting and Decision Tree, showing it's a powerful model with a very low RMSE.
Voting Regressor	-	0.3601	99.56	Combines predictions of all models and provides moderate RMSE. Does not outperform XGBoost or Gradient Boosting but balances strengths of each model. 's different.

Concrete Compressive Strength Dataset¶

Author- Mubin Modi¶

Overview¶

Dataset Attributes¶

Specific Insights¶

Trends in Compressive Strength Over Different Curing Ages¶

Trends in Compressive Strength Over Different Curing Ages¶

Comparison with Other Studies on Curing Age and Compressive Strength¶

Specific Insights¶

Additional Observations¶

Correlation Matrix¶

Conclusions from Correlation Matrix¶

Checking for Outliers¶

Data Prep , Model Training and Model Evaluation¶

Summary of Best Models, RMSE, and Accuracy¶

Conclusions:¶